import requests
import json
from tqdm import trange
import pymongo
import time
from bs4 import BeautifulSoup as Soup

url = 'http://119.23.8.25/rules/'
headers = {
    "Host": "119.23.8.25",
    "Connection": "keep-alive",
    "Pragma": "no-cache",
    "Cache-Control": "no-cache",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/87.0.4280.67 Safari/537.36 Edg/87.0.664.55",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,"
              "application/signed-exchange;v=b3;q=0.9",
    "Referer": "http://119.23.8.25/rules/",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
}
client = pymongo.MongoClient()
db = client.robomaster
col = db.rm2021


def request_once():
    content = requests.get(url, headers=headers).content
    soup = Soup(content, 'html.parser')
    entries_text = soup.select('text[style="font-weight:bold"]')
    # print(len(entries_text), entries_text)
    entries_options = soup.find_all(attrs={'class': 'radio'})
    # print(len(entries_options), entries_options)
    collection = {}
    for i in range(len(entries_text)):
        problem = entries_text[i].get_text()
        if '、' in problem:
            problem = problem.split('、')[-1]
        options = entries_options[i * 4: i * 4 + 4]
        index = options[0].select("input")[0].attrs['index']
        options = [{
            'text': e.get_text(),
            'value': e.select("input")[0].attrs['value']
        } for e in options]
        # print(problem)
        # print(options)
        result = {
            'problem': problem,
            'index': index,
            'options': options
        }
        # print(result)
        collection[index] = result
    return collection


def main():
    target_count = 500
    sleep_time = 1.5
    for i in trange(target_count):
        collection = request_once()
        for c_index in collection:
            c = collection[c_index]
            col.update_one({'index': {"equ": c_index}}, {"$setOnInsert": c}, upsert=True)
        time.sleep(sleep_time)


def find_all():
    # 找到最大的
    max_index = max(map(int, col.find().distinct("index")))
    # .sort([("index", -1)]).limit(3)
    ps = []
    print('max index:', max_index)
    for i in trange(max_index):
        data = list(col.find({'index': str(i)}))
        if len(data) == 0:
            continue
        p = data[0]
        if '_id' in p:
            del p['_id']
        # print(p)
        ps.append(p)
    with open("all.json", 'w', encoding='utf8') as f:
        t = json.dumps(ps)
        f.write(t)
    print('done')


if __name__ == '__main__':
    # request_once()
    # main()
    find_all()
